﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;
using System.Diagnostics;

namespace CrawleWeb
{
    class Program
    {
        static Int32 RequestCount = 0;
        static Int32 WriteCount = 0;
        static readonly String BasicUrl = "http://www.stats.gov.cn/tjbz/cxfldm/2012/";
        static Queue<RequestUlr> requestWebQueue = new Queue<RequestUlr>();
        static Queue<WebContent> handleContentQueue = new Queue<WebContent>();
        static ManualResetEvent requestWebWait = new ManualResetEvent(false);
        static ManualResetEvent handleContentWait = new ManualResetEvent(false);
        static void Main(string[] args)
        {
            Thread requestWebThread;
            Thread handleContentThread;
            
            File.Delete("行政区划.txt");
            File.Delete("请求出错地址.txt");
            Int32 ThreadCount = 0;
            Console.Write("你的CPU为{0}核，请输入Http请求线程数：",Environment.ProcessorCount);
            Int32.TryParse(Console.ReadLine(), out ThreadCount);
            Stopwatch stopwatch = new Stopwatch();
            System.Timers.Timer timer = new System.Timers.Timer(5000);
            timer.Elapsed += (sender, e) =>
            {
                var Count=requestWebQueue.Count;
                Console.WriteLine("{0} 保存{1}条数据 执行{2}次请求 未处理请求{3}个 剩余时间{4}秒", stopwatch.Elapsed.ToString(@"hh\:mm\:ss"), WriteCount,
                        RequestCount, Count, (Int32)(Count * stopwatch.Elapsed.TotalSeconds / RequestCount));
                if (handleContentQueue.Count == 0&&requestWebQueue.Count==0)
                {
                    Console.WriteLine("任务执行完毕！");
                }
            };
            timer.Start();
            stopwatch.Start();

            Random rd = new Random();
            Thread.Sleep(rd.Next(100, 500));
            requestWebQueue.Enqueue(new RequestUlr {Level=1,Url=BasicUrl });
            for (Int32 i = 0; i < ThreadCount; i++)
            {
                requestWebThread = new Thread(new ThreadStart(RequestWeb));
                requestWebThread.Name = "RequestWebThread"+i;
                requestWebThread.Start();
                Thread.Sleep(rd.Next(100, 100*ThreadCount));
            }
            handleContentThread = new Thread(new ThreadStart(HandleContent));
            handleContentThread.Start();
        }

        /// <summary>
        /// RequestWeb
        /// </summary>
        static void RequestWeb()
        {
            while (true)
            {
                RequestUlr item = null;
                lock (requestWebQueue)
                {
                    if (requestWebQueue.Count > 0)
                    {
                        item = requestWebQueue.Dequeue();
                    }
                }
                if (item == null)
                {
                    //Console.WriteLine("RequestWebThread空闲");
                    Thread.Sleep(100);
                    //Console.WriteLine("requestWebWait reset:");
                    //requestWebWait.Reset();
                }
                else
                {
                    try
                    {
                        /*
                        WebClient client = new WebClient();
                        client.Encoding = Encoding.GetEncoding("GB2312");
                        client.UseDefaultCredentials = false;
                        /*client.DownloadStringCompleted +=new DownloadStringCompletedEventHandler((obj,eventArgs)=>
                        {
                            //handleContentWait.Reset();
                            lock (handleContentQueue)
                            {
                                handleContentQueue.Enqueue(new WebContent(item.Level,eventArgs.Result));
                            }
                            //handleContentWait.Set();
                            client.Dispose();
                        });* /
                        string content= client.DownloadString(new Uri(item.Url));
                        lock (handleContentQueue)
                        {
                            handleContentQueue.Enqueue(new WebContent(item.Level, content));
                        }
                        */
                        #region HttpWebRequest Mode

                        var webRequest = (HttpWebRequest)WebRequest.Create(item.Url);
                        webRequest.UseDefaultCredentials = false;
                        webRequest.Method = "GET";
                        webRequest.ContentLength = 0;
                        webRequest.AllowAutoRedirect = false;
                        using (var webResponse = (HttpWebResponse)webRequest.GetResponse())
                        {
                            //webResponse.ContentEncoding
                            using (StreamReader reader = new StreamReader(webResponse.GetResponseStream(), Encoding.GetEncoding("GB2312")))
                            {
                                var content = reader.ReadToEnd();
                                lock (handleContentQueue)
                                {
                                    handleContentQueue.Enqueue(new WebContent(item.Level, content));
                                }
                            }
                        }

                        #endregion
                        //Console.WriteLine("RequestUrl: " + item.Url);
                        //Console.WriteLine(Thread.CurrentThread.Name);
                        Interlocked.Increment(ref RequestCount);
                    }
                    catch (Exception)
                    {
                        using (StreamWriter writer = new StreamWriter("请求出错地址.txt", true))
                        {
                            writer.WriteLine(item.Url);
                        }
                        Console.WriteLine("请求出错地址：" + item.Url);
                    }
                }
                //requestWebWait.WaitOne();
            }
        }

        /// <summary>
        /// HandleContent
        /// </summary>
        static void HandleContent()
        {
            while (true)
            {
                WebContent item = null;
                lock (handleContentQueue)
                {
                    if (handleContentQueue.Count > 0)
                    {
                        item = handleContentQueue.Dequeue();
                    }
                }
                if (item == null)
                {
                    //Console.WriteLine("HandleContentThread空闲");
                    Thread.Sleep(100);
                    //Console.WriteLine("handleContentWait reset:");
                    //handleContentWait.Reset();
                }
                else
                {
                    try
                    {
                        var Divisions = GetDivisionCode(item.Content, item.Level);
                        using (StreamWriter writer = new StreamWriter("行政区划.txt", true))
                        {
                            foreach (var d in Divisions)
                            {
                                writer.WriteLine("{0},{1},{2},{3}", d.Name, d.TownCode, d.Code, item.Level);
                                Interlocked.Increment(ref WriteCount);
                            }
                            //Console.WriteLine("SaveWebContent: " + pair.Key);
                        }
                        //requestWebWait.Reset();
                        lock (requestWebQueue)
                        {
                            foreach (var d in Divisions)
                            {
                                if (d.HaveSub)
                                {
                                    requestWebQueue.Enqueue(new RequestUlr
                                    {
                                        Level = item.Level + 1,
                                        Url = GetNextLevelUrl(d.Code, item.Level),
                                    });
                                }
                            }
                        }
                        //requestWebWait.Set();
                    }
                    catch (Exception ex)
                    {
                        Console.WriteLine("保存数据或添加队列异常：" + ex.Message);
                    }
                }
                //handleContentWait.WaitOne();
            }
        }

        /// <summary>
        /// 获取下级Url
        /// </summary>
        /// <param name="Code">上级区划代码</param>
        /// <param name="Level">上级级数</param>
        /// <returns></returns>
        static String GetNextLevelUrl(String Code, Int32 Level)
        {
            if (String.IsNullOrEmpty(Code))
                return String.Empty;
            StringBuilder Sb = new StringBuilder(BasicUrl);
            switch (Level)
            {
                case 1:
                    Sb.Append(Code.Substring(0, 2));
                    Sb.Append(".html");
                    break;
                case 2:
                    Sb.Append(Code.Substring(0, 2));
                    Sb.Append("/");
                    Sb.Append(Code.Substring(0, 4));
                    Sb.Append(".html");
                    break;
                case 3:
                    Sb.Append(Code.Substring(0, 2));
                    Sb.Append("/");
                    Sb.Append(Code.Substring(2, 2));
                    Sb.Append("/");
                    Sb.Append(Code.Substring(0, 6));
                    Sb.Append(".html");
                    break;
                case 4:
                    Sb.Append(Code.Substring(0, 2));
                    Sb.Append("/");
                    Sb.Append(Code.Substring(2, 2));
                    Sb.Append("/");
                    Sb.Append(Code.Substring(4, 2));
                    Sb.Append("/");
                    Sb.Append(Code.Substring(0, 9));
                    Sb.Append(".html");
                    break;
                default:
                    return BasicUrl;
            }
            return Sb.ToString();
        }

        /// <summary>
        /// 获取区划代码
        /// </summary>
        /// <param name="Content"></param>
        /// <param name="Level"></param>
        /// <returns></returns>
        static IList<DivisionCode> GetDivisionCode(String Content, Int32 Level)
        {
            IList<DivisionCode> _items = new List<DivisionCode>();
            if (Level == 1)
            {
                /*
                this._items = (new Dictionary<String, String>
                {
                    {"北京","110000000000"},{"天津","120000000000"},{"河北","130000000000"}, {"山西","140000000000"}, 
                    {"内蒙古","150000000000"}, {"辽宁","210000000000"}, {"吉林","220000000000"}, {"黑龙江","230000000000"}, 
                    {"上海","310000000000"}, {"江苏","320000000000"}, {"浙江","330000000000"}, {"安徽","340000000000"},
                    {"福建","350000000000"}, {"江西","360000000000"}, {"山东","370000000000"}, {"河南","410000000000"}, 
                    {"湖北","420000000000"}, {"湖南","430000000000"}, {"广东","440000000000"}, {"广西","450000000000"}, 
                    {"海南","460000000000"}, {"重庆","500000000000"}, {"四川","510000000000"}, {"贵州","520000000000"}, 
                    {"云南","530000000000"}, {"西藏","540000000000"}, {"陕西","610000000000"}, {"甘肃","620000000000"}, 
                    {"青海","630000000000"}, {"宁夏","640000000000"}, {"新疆","650000000000"}
                }).ToDictionary(m => m.Key, n => new DivisionCode { Name = n.Key, Code = n.Value, HaveSub = true });
                */
                String pattern = "<tr class='provincetr'>(<td width='12.5%'><a href='(?<Href>.+?)'>(?<Name>.+?)</a></td> )+(<td>&nbsp;</td>)?</tr>";
                foreach (Match m in Regex.Matches(Content, pattern, RegexOptions.IgnoreCase))
                {
                    for (var i = 0; i < m.Groups["Name"].Captures.Count; i++)
                    {
                        _items.Add(new DivisionCode
                        {
                            Name = m.Groups["Name"].Captures[i].Value,
                            Code = m.Groups["Href"].Captures[i].Value.Substring(0, 2) + "0000000000",
                            TownCode = String.Empty,
                            HaveSub = true
                        });
                    }
                }
            }
            else
            {
                String[] trName = { "provincetr", "citytr", "countytr", "towntr", "villagetr" };
                String pattern = ("<tr class='{0}'><td>(<a.+?href='(?<Href>.+?)'>)?(?<Code>.+?)(</a>)?</td>(<td>(?<TownCode>.{0,3})</td>)?<td>(<a.+?>)?(?<Name>.+?)(</a>)?</td></tr>").Replace("{0}"
                    , trName[Level - 1]);
                foreach (Match m in Regex.Matches(Content, pattern, RegexOptions.IgnoreCase))
                {
                    _items.Add(new DivisionCode
                    {
                        Name = m.Groups["Name"].Value,
                        Code = m.Groups["Code"].Value,
                        TownCode = m.Groups["TownCode"].Value,
                        HaveSub = String.IsNullOrEmpty(m.Groups["Href"].Value) ? false : true
                    });
                }
            }
            return _items;
        }
    }
    class RequestUlr
    {
        public String Url { get; set; }
        public Int32 Level { get; set; }
    }
    class WebContent
    {
        public WebContent(Int32 level, String content)
        {
            this.Level = level;
            this.Content = content;
        }
        public String Content { get; set; }
        public Int32 Level { get; set; }
    }
    class DivisionCode
    {
        /// <summary>
        /// 区划名称
        /// </summary>
        public String Name { get; set; }
        /// <summary>
        /// 区划代码
        /// </summary>
        public String Code { get; set; }
        /// <summary>
        /// 城乡分类代码
        /// </summary>
        public String TownCode { get; set; }
        /// <summary>
        /// 是否有子级
        /// </summary>
        public Boolean HaveSub { get; set; }
    }
}
